#!/usr/bin/perl
use strict;

my $faq_file= "C:\\DOCUME~1\\Administrator\\workspace\\smsindex\\data\\FAQ_all.txt";
open(FAQ,$faq_file) || die('cant open $faq_file');
my $stop_file ="C:\\DOCUME~1\\Administrator\\workspace\\smsindex\\data\\stopwords";
open(STOPW,$stop_file) || die('cant open $stop_file');
my @stop = <STOPW>;
my $out_faq = "C:\\DOCUME~1\\Administrator\\workspace\\smsindex\\data\\FAQ_nostop.txt";
open(OUT,">$out_faq") || die('cant open $out_faq');

for(my $i=0;$i<@stop;$i++){
	chomp($stop[$i]);
}

while(<FAQ>){
	chomp($_);
	my @words = split("[ ]+",$_);
	foreach my $wd (@words){
		#print $wd."\n";
		my $found = 'false';
		foreach (@stop){
			if((lc($_) eq lc($wd))|| (lc($_)."s" eq lc($wd)) || (lc($_)."ies" eq lc($wd))){
				$found = 'true';
			}	
		}
		if($found eq 'false'){ print OUT " ".lc($wd)}		
	}
	print OUT "\n";
}
